import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from collections import Counter
import paddlehub as hub
import paddle
from sklearn.model_selection import train_test_split
from paddlehub.datasets.base_nlp_dataset import TextClassificationDataset
df=pd.read_excel('moods_classify8_unprocessed.xlsx')
df.head(10)
df.info()
df.isnull().any()
df[df.isnull().values==True]
df.dropna(subset=['text','label'],axis=0,how='any',inplace=True)
df.isnull().any()
df[df.duplicated('text')]
df.drop_duplicates(subset='text',keep='first',inplace=True)
df.duplicated('text').any()
plt.boxplot(x=df.label,
           whis=1.5,
           widths=0.8,
           patch_artist=True,
           showmeans=True,
           boxprops={'facecolor':'steelblue'},
           flierprops={'markerfacecolor':'red','markeredgecolor':'red','markersize':4},
           meanprops={'marker':'D','markerfacecolor':'black','markersize':4},
           medianprops={'linestyle':'--','color':'orange'},
           labels=[''])
plt.show()
Q1=df.label.quantile(q=0.25)
Q3=df.label.quantile(q=0.75)
low_whisker=Q1-1.5*(Q3-Q1)
up_whisker=Q3-1.5*(Q3-Q1)
df2=df.label[(df.label>up_whisker)|(df.label<low_whisker)]
print(Counter(df2))
df[df['label']==9.0]
df.drop((df[df['label']==9.0]).index,inplace=True)
(df['label']==9.0).any()
df.info()
df['text'].str.len().describe()
train_labled=df[['label','text']]
train,test=train_test_split(train_labled,test_size=0.2,random_state=2021)
train.to_csv('train.txt',index=False,header=False,sep='\t')
test.to_csv('test.txt',index=False,header=False,sep='\t')
txt_list=['train.txt','test.txt']
I=0
for file in txt_list:
    with open(file,'r')as f:
        I+=len(f.readlines())
print("拆分后的数据量为: ",I)
from paddlehub.datasets.base_nlp_dataset import TextClassificationDataset
class MyDataset(TextClassificationDataset):
    base_path='data'
    label_list=['0.0','1.0','2.0','3.0','4.0','5.0','6.0','7.0']
    def __init__(self,tokenizer,max_seq_len:int=128,mode:str='train'):
        if mode=='train':
            data_file='train.txt'
        elif mode=='test':
            data_file='test.txt'
        else:
            data_file='dev.txt'
        super().__init__(
        base_path=self.base_path,
        tokenizer=tokenizer,
        max_seq_len=max_seq_len,
        mode=mode,
        data_file=data_file,
        label_list=self.label_list,
        is_file_with_header=False)
model=hub.Module(name='ernie_tiny',task='seq-cls',num_classes=len(MyDataset.label_list))
tokenizer=model.get_tokenizer()
train_dataset=MyDataset(tokenizer)
test_dataset=MyDataset(tokenizer,mode='test')